#!/usr/bin/python
# -*- coding: UTF-8 -*-
import requests
import os

from requests import HTTPError

from spider.requests_helper import Request
from lxml import html
from selenium import webdriver

def get_page_total():
    url = "http://yyrr12.com/Video/21-1/"
    # 获得html内容
    html_text = ''
    retry_count = 5
    for i in range(1, retry_count):
        if html_text != '':
            break
        try:
            html_text = Request().get_html_text(url)
        except HTTPError:
            print("获得html内容发送错误进行重试")

    # 获得xpath对象
    selector = html.fromstring(html_text)

    page_total = selector.xpath('//button[@class="page_btn mac_page_go"]/@data-total')

    total = page_total[0]
    return total


def spider(page):
    """ 爬取yyrr12网站视频的访问地址 """
    base_url = "http://yyrr12.com"
    url = base_url + "/Video/21-{page}/".format(page=page)
    # 获得html内容
    html_text = ''
    retry_count = 5
    for i in range(1, retry_count):
        if html_text != '':
            break
        try:
            html_text = Request().get_html_text(url)
        except HTTPError:
            print("获得html内容发送错误进行重试")

    # 获得xpath对象
    selector = html.fromstring(html_text)

    li_list = selector.xpath('//ul[@class="img-list-data"]/div/li')

    content_list = []
    for li in li_list:
        content_info = {}
        content_info['href'] = base_url + (li.xpath('a/@href')[0])
        content_info['title'] = str(li.xpath('a/@title')[0])
        content_info['src'] = str(li.xpath('a/img/@src')[0])
        content_list.append(content_info)

    return content_list


if __name__ == '__main__':
    start_page = 1
    end_page = int(get_page_total()) + 1
    file = open('G:/yyrr12访问地址3.txt', 'w', encoding='gbk')
    for i in range(start_page, end_page):
        file.write("第{page}页\n".format(page=i))
        content_list = spider(i)
        for content in content_list:
            try:
                file.write(content['title'] + "\n" + content['href'] + "\n")
            except UnicodeEncodeError:
                print(content['title'] + " : " + content['href'] + " 不能写入")
        print("下载{page}页成功".format(page=i))
    file.close()
